#!/usr/bin/env python

import wsgiref.handlers
from google.appengine.ext import webapp
from google.appengine.api.labs import taskqueue

from google.appengine.ext import db
from schema import Document, Fetch, key_name_from_checksum

import re, logging

class LinkFinder(webapp.RequestHandler):
  def post(self):
    computationid = self.request.get("computationid")
    logging.debug("This is on behalf of computation %s." % computationid)
    fetch_key = self.request.get("fetch_key")
    logging.debug("I'm going to try to parse fetch %s." % fetch_key)
    fetch = Fetch.get(db.Key(fetch_key))
    content = fetch.content

    anchors = []
    pattern = re.compile(r'<a href="([^"]+)"')
    for line in content.split('>'):
      match = pattern.search(line)
      if match:
        anchors.append(unicode(match.group(1), encoding="utf-8"))

    logging.info("I found %i anchors." % len(anchors))

    for url in anchors:
      logging.debug("I'm adding a task to add a link to %s" % url)
      task = taskqueue.Task(
        url='/add-link',
        params={
          'computationid': computationid,
          'fetch_key': fetch_key,
          'url': url,
        }
      )
      task.add(queue_name='parse')


def main():
  logging.getLogger().setLevel(logging.DEBUG)
  wsgiref.handlers.CGIHandler().run(webapp.WSGIApplication([
    ('/find-links', LinkFinder),
  ]))

if __name__ == '__main__':
  main()
